# encoding: utf-8

import json
from tqdm import tqdm
from typing import List, Dict, Any

from utils import auto_binding, ners

with open("./datas/datas.json", "r", encoding="utf-8") as f:
    datas = json.load(f)

# datas = datas[:20]
filter_ = set()
datas2 = []
for data in datas:
    _id = data.get("_id")
    if _id in filter_:
        continue
    filter_.add(_id)
    datas2.append(data)

batch_size = 128

data_lists = [datas2[i: i + batch_size] for i in range(0, len(datas2), batch_size)]
del datas
del filter_


# 批量ner
def ner_batch(data_list: List[Dict[str, Any]], key: str) -> List[Dict[str, Any]]:
    texts = []
    result = []
    for data in data_list:
        text = data.get(key, "")
        texts.append(text)
    ner_result = ners.extract_batch(texts)
    for one_data, one_ner in zip(data_list, ner_result):
        one_data[f"{key}_ner"] = one_ner
        result.append(one_data)

    return result


ner_result = []
maybe_err = []

for data_list in tqdm(data_lists, desc="抽取"):
    result1 = ner_batch(data_list, "skuName")
    result2 = ner_batch(result1, "bindSkuName")
    ner_result += result2

for data in tqdm(ner_result, desc="判断"):
    brand = data.get("brand", "")
    sku_name = data.get("skuName", "")
    bind_sku_name = data.get("bindSkuName")

    material_ner_data = data.get("skuName_ner", {})
    bind_ner_data = data.get("bindSkuName_ner", {})

    judge_result = auto_binding.is_same_sku(data, material_ner_data, bind_ner_data, bind_sku_name)

    if judge_result == "true":
        continue
    else:
        data.pop("skuName_ner", None)
        data.pop("bindSkuName_ner", None)
        maybe_err.append(data)

with open("./maybe_err1.json", "w", encoding="utf-8") as f:
    json.dump(maybe_err, f, ensure_ascii=False)

# for maybe in maybe_err:
#     print(maybe)

print("maybe_err.length=", len(maybe_err))
